<html>
 <head>
  <meta charset="utf-8"/>
  <meta content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no" name="viewport"/>
  <title>
   机器学习中导数最优化方法(基础篇)  | 数螺 | NAUT IDEA
  </title>
  <link href="http://cdn.bootcss.com/bootstrap/3.3.6/css/bootstrap-theme.min.css" rel="stylesheet"/>
  <link href="http://cdn.bootcss.com/bootstrap/3.3.6/css/bootstrap.min.css" rel="stylesheet"/>
  <style type="text/css">
   #xmain img {
                  max-width: 100%;
                  display: block;
                  margin-top: 10px;
                  margin-bottom: 10px;
                }

                #xmain p {
                    line-height:150%;
                    font-size: 16px;
                    margin-top: 20px;
                }

                #xmain h2 {
                    font-size: 24px;
                }

                #xmain h3 {
                    font-size: 20px;
                }

                #xmain h4 {
                    font-size: 18px;
                }


                .header {
	           background-color: #0099ff;
	           color: #ffffff;
	           margin-bottom: 20px;
	        }

	        .header p {
                  margin: 0px;
                  padding: 10px 0;
                  display: inline-block;  
                  vertical-align: middle;
                  font-size: 16px;
               }

               .header a {
                 color: white;
               }

              .header img {
                 height: 25px;
              }
  </style>
  <script src="http://cdn.bootcss.com/jquery/3.0.0/jquery.min.js">
  </script>
  <script src="http://nautstatic-10007657.file.myqcloud.com/static/css/readability.min.js" type="text/javascript">
  </script>
  <script type="text/javascript">
   $(document).ready(function() {
                 var loc = document.location;
                 var uri = {
                  spec: "http://dataunion.org/11752.html",
                  host: "http://dataunion.org",
                  prePath: "http://dataunion.org",
                  scheme: "http",
                  pathBase: "http://dataunion.org/"
                 };
    
                 var documentClone = document.cloneNode(true);
                 var article = new Readability(uri, documentClone).parse();
     
                 document.getElementById("xmain").innerHTML = article.content;
                });
  </script>
  <!-- 1466460900: Accept with keywords: (title(0.333333333333):社区,导数,基础,数盟,方法,最优化, topn(0.6):导数,等值线,域法,数盟,二阶,利用,过程,人工智能,局部,阻尼,计算,文章,方法,向量,特征值,椭圆,函数,公式,矩阵,目标,算法,偏导,极小值,优化,最优化,梯度,代码,最小值,方向,共轭).-->
 </head>
 <body onload="">
  <div class="header">
   <div class="container">
    <div class="row">
     <div class="col-xs-6 col-sm-6 text-left">
      <a href="/databee">
       <img src="http://nautidea-10007657.cos.myqcloud.com/logo_white.png"/>
      </a>
      <a href="/databee">
       <p>
        数螺
       </p>
      </a>
     </div>
     <div class="hidden-xs col-sm-6 text-right">
      <p>
       致力于数据科学的推广和知识传播
      </p>
     </div>
    </div>
   </div>
  </div>
  <div class="container text-center">
   <h1>
    机器学习中导数最优化方法(基础篇)
   </h1>
  </div>
  <div class="container" id="xmain">
   ﻿﻿
   <title>
    机器学习中导数最优化方法(基础篇) | 数盟社区
   </title>
   <!-- All in One SEO Pack 2.2.7.6.2 by Michael Torbert of Semper Fi Web Design[32,64] -->
   <!-- /all in one seo pack -->
   <!--
<div align="center">
<a href="http://strata.oreilly.com.cn/hadoop-big-data-cn?cmp=mp-data-confreg-home-stcn16_dataunion_pc" target="_blank"><img src="http://dataunion.org/wp-content/uploads/2016/05/stratabj.jpg"/ ></a>
</div>
-->
   <header id="header-web">
    <div class="header-main">
     <hgroup class="logo">
      <h1>
       <a href="http://dataunion.org/" rel="home" title="数盟社区">
        <img src="http://dataunion.org/wp-content/themes/yzipi/images/logo.png"/>
       </a>
      </h1>
     </hgroup>
     <!--logo-->
     <nav class="header-nav">
      <ul class="menu" id="menu-%e4%b8%bb%e8%8f%9c%e5%8d%95">
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-has-children menu-item-71" id="menu-item-71">
        <a href="http://dataunion.org/category/events" title="events">
         活动
        </a>
        <ul class="sub-menu">
         <li class="menu-item menu-item-type-post_type menu-item-object-page menu-item-22457" id="menu-item-22457">
          <a href="http://dataunion.org/2016timeline">
           2016档期
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-22459" id="menu-item-22459">
          <a href="http://dataunion.org/category/parterc">
           合作会议
          </a>
         </li>
        </ul>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category current-post-ancestor menu-item-has-children menu-item-20869" id="menu-item-20869">
        <a href="http://dataunion.org/category/tech" title="articles">
         文章
        </a>
        <ul class="sub-menu">
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-20867" id="menu-item-20867">
          <a href="http://dataunion.org/category/tech/base" title="base">
           基础架构
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category current-post-ancestor current-menu-parent current-post-parent menu-item-3302" id="menu-item-3302">
          <a href="http://dataunion.org/category/tech/ai" title="ai">
           人工智能
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-3303" id="menu-item-3303">
          <a href="http://dataunion.org/category/tech/analysis" title="analysis">
           数据分析
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-21920" id="menu-item-21920">
          <a href="http://dataunion.org/category/tech/dm">
           数据挖掘
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-3314" id="menu-item-3314">
          <a href="http://dataunion.org/category/tech/viz" title="viz">
           可视化
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-3305" id="menu-item-3305">
          <a href="http://dataunion.org/category/tech/devl" title="devl">
           编程语言
          </a>
         </li>
        </ul>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-has-children menu-item-20876" id="menu-item-20876">
        <a href="http://dataunion.org/category/industry">
         行业
        </a>
        <ul class="sub-menu">
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-16328" id="menu-item-16328">
          <a href="http://dataunion.org/category/industry/case" title="case">
           行业应用
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-2112" id="menu-item-2112">
          <a href="http://dataunion.org/category/industry/demo" title="demo">
           Demo展示
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-21562" id="menu-item-21562">
          <a href="http://dataunion.org/category/industry/news">
           行业资讯
          </a>
         </li>
        </ul>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-311" id="menu-item-311">
        <a href="http://dataunion.org/category/sources" title="sources">
         资源
        </a>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-20870" id="menu-item-20870">
        <a href="http://dataunion.org/category/books" title="book">
         图书
        </a>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-21363" id="menu-item-21363">
        <a href="http://dataunion.org/category/training">
         课程
        </a>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-has-children menu-item-21853" id="menu-item-21853">
        <a href="http://dataunion.org/category/jobs">
         职位
        </a>
        <ul class="sub-menu">
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-22050" id="menu-item-22050">
          <a href="http://dataunion.org/category/career">
           职业规划
          </a>
         </li>
        </ul>
       </li>
      </ul>
     </nav>
     <!--header-nav-->
    </div>
   </header>
   <!--header-web-->
   <div id="main">
    <div id="soutab">
     <form action="http://dataunion.org/" class="search" method="get">
     </form>
    </div>
    <div id="container">
     <nav id="mbx">
      当前位置：
      <a href="http://dataunion.org">
       首页
      </a>
      &gt;
      <a href="http://dataunion.org/category/tech">
       文章
      </a>
      &gt;
      <a href="http://dataunion.org/category/tech/ai">
       人工智能
      </a>
      &gt;  正文
     </nav>
     <!--mbx-->
     <article class="content">
      <header align="centre" class="contenttitle">
       <div class="mscc">
        <h1 class="mscctitle">
         <a href="http://dataunion.org/11752.html">
          机器学习中导数最优化方法(基础篇)
         </a>
        </h1>
        <address class="msccaddress ">
         <em>
          4,156 次阅读 -
         </em>
         <a href="http://dataunion.org/category/tech/ai" rel="category tag">
          人工智能
         </a>
        </address>
       </div>
      </header>
      <div class="content-text">
       <div id="cnblogs_post_body">
        <h2>
         作者：daniel-D
        </h2>
        <h2>
         1. 前言
        </h2>
        <p>
         熟悉机器学习的童鞋都知道，优化方法是其中一个非常重要的话题，最常见的情形就是利用目标函数的导数通过多次迭代来求解无约束最优化问题。实现简单，coding 方便，是训练模型的必备利器之一。这篇博客主要总结一下使用导数的最优化方法的几个基本方法，梳理梳理相关的数学知识，本人也是一边写一边学，如有问题，欢迎指正，共同学习，一起进步。
        </p>
        <p>
        </p>
        <h2>
         2. 几个数学概念
        </h2>
        <h3>
         1) 梯度（一阶导数）
        </h3>
        <p>
         考虑一座在 (x1, x2) 点高度是 f(x1, x2) 的山。那么，某一点的梯度方向是在该点坡度最陡的方向，而梯度的大小告诉我们坡度到底有多陡。注意，梯度也可以告诉我们不在最快变化方向的其他方向的变化速度（二维情况下，按照梯度方向倾斜的圆在平面上投影成一个椭圆）。对于一个含有 n 个变量的标量函数，即函数输入一个 n 维 的向量，输出一个数值，梯度可以定义为：
        </p>
        <p>
        </p>
        <center>
         <img src="http://dataunion.org/wp-content/uploads/2015/03/3897cde8f174309a9fc398795411769d.png"/>
        </center>
        <p>
        </p>
        <h3>
         2) Hesse 矩阵（二阶导数）
        </h3>
        <p>
         Hesse 矩阵常被应用于
         <strong>
          牛顿法解决的大规模优化问题
         </strong>
         (后面会介绍)，主要形式如下：
        </p>
        <p>
        </p>
        <center>
         <img src="http://dataunion.org/wp-content/uploads/2015/03/75dc4e662a991741dc6d0314b6c8a7d2.png"/>
        </center>
        当 f(x) 为二次函数时，梯度以及 Hesse 矩阵很容易求得。二次函数可以写成下列形式：
        <p>
        </p>
        <p>
        </p>
        <center>
         <img src="http://latex.codecogs.com/gif.latex?f(x)=\frac{1}{2}x^TAx%20+%20b^Tx+c"/>
        </center>
        其中 A 是 n 阶对称矩阵，b 是 n 维列向量， c 是常数。f(x) 梯度是 Ax+b, Hesse 矩阵等于 A。
        <p>
        </p>
        <h3>
         3) Jacobi 矩阵
        </h3>
        <p>
         Jacobi 矩阵实际上是向量值函数的梯度矩阵，假设F:Rn→Rm 是一个从n维欧氏空间转换到m维欧氏空间的函数。这个函数由m个实函数组成:
         <img src="http://latex.codecogs.com/gif.latex?%20y(x)%20=%20[y_1(x_1,%20...,%20x_n),\cdots%20,%20y_m(x_1,%20...,x_n)]"/>
         。这些函数的偏导数(如果存在)可以组成一个m行n列的矩阵(m by n)，这就是所谓的雅可比矩阵：
        </p>
        <p>
        </p>
        <center>
         <img src="http://dataunion.org/wp-content/uploads/2015/03/8bbb6b0c9c437a682aa7b4ecd1b6d0f2.png"/>
        </center>
        总结一下,
        <p>
        </p>
        <p>
         <strong>
          a)
         </strong>
         如果 f(x) 是一个标量函数，那么雅克比矩阵是一个向量，等于 f(x) 的梯度， Hesse 矩阵是一个二维矩阵。如果 f(x) 是一个向量值函数，那么Jacobi 矩阵是一个二维矩阵，Hesse 矩阵是一个三维矩阵。
        </p>
        <p>
         <strong>
          b)
         </strong>
         梯度是 Jacobian 矩阵的特例，梯度的 jacobian 矩阵就是 Hesse 矩阵（一阶偏导与二阶偏导的关系）。
        </p>
        <p>
        </p>
        <h2>
         3. 优化方法
        </h2>
        <h3>
         1) Gradient Descent
        </h3>
        <p>
         Gradient descent 又叫 steepest descent，是利用一阶的梯度信息找到函数局部最优解的一种方法，也是机器学习里面最简单最常用的一种优化方法。Gradient descent 是 line search 方法中的一种，主要迭代公式如下：
        </p>
        <p>
        </p>
        <center>
         <img src="http://latex.codecogs.com/gif.latex?x_{k+1}%20=%20x_k%20+%20\alpha%20_k\mathbf{p}_k"/>
        </center>
        其中，
        <img src="http://latex.codecogs.com/gif.latex?\mathbf{p}_k"/>
        是第 k 次迭代我们选择移动的方向，在 steepest descent 中，移动的方向设定为梯度的负方向，
        <img src="http://latex.codecogs.com/gif.latex?\alpha%20_k"/>
        是第 k 次迭代用 line search 方法选择移动的距离，每次移动的距离系数可以相同，也可以不同，有时候我们也叫学习率（learning rate）。在数学上，移动的距离可以通过 line search 令导数为零找到该方向上的最小值，但是在实际编程的过程中，这样计算的代价太大，我们一般可以将它设定位一个常量。考虑一个包含三个变量的函数
        <img src="http://latex.codecogs.com/gif.latex?f(\mathbf{x})=0.5x_1^2+0.2x_2^2+0.6x_3^2"/>
        ，计算梯度得到
        <img src="http://latex.codecogs.com/gif.latex?\nabla%20f(\mathbf{x})=(x_1,0.4x_2,1.2x_3)"/>
        。设定 learning rate = 1，算法代码如下：
        <p>
        </p>
        <div class="cnblogs_code">
         <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
         <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-57686ae078253193916527" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
          <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
           <span class="crayon-title">
           </span>
           <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
            <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
             <div class="crayon-button-icon">
             </div>
            </div>
            <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
             <div class="crayon-button-icon">
             </div>
            </div>
            <div class="crayon-button crayon-wrap-button" title="切换自动换行">
             <div class="crayon-button-icon">
             </div>
            </div>
            <div class="crayon-button crayon-expand-button" title="点击展开代码">
             <div class="crayon-button-icon">
             </div>
            </div>
            <div class="crayon-button crayon-copy-button" title="复制代码">
             <div class="crayon-button-icon">
             </div>
            </div>
            <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
             <div class="crayon-button-icon">
             </div>
            </div>
           </div>
          </div>
          <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
          </div>
          <div class="crayon-plain-wrap">
           <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
            # Code from Chapter 11 of Machine Learning: An Algorithmic Perspective
# by Stephen Marsland (http://seat.massey.ac.nz/personal/s.r.marsland/MLBook.html)

# Gradient Descent using steepest descent

from numpy import *

def Jacobian(x):
    return array([x[0], 0.4*x[1], 1.2*x[2]])

def steepest(x0):

    i = 0 
    iMax = 10
    x = x0
    Delta = 1
    alpha = 1

    while i&amp;lt;iMax and Delta&amp;gt;10**(-5):
        p = -Jacobian(x)
        xOld = x
        x = x + alpha*p
        Delta = sum((x-xOld)**2)
        print 'epoch', i, ':'
        print x, '\n'
        i += 1

x0 = array([-2,2,-2])
steepest(x0)
           </textarea>
          </div>
          <div class="crayon-main" style="">
           <table class="crayon-table">
            <tbody>
             <tr class="crayon-row">
              <td class="crayon-nums " data-settings="show">
               <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                <div class="crayon-num" data-line="crayon-57686ae078253193916527-1">
                 1
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078253193916527-2">
                 2
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078253193916527-3">
                 3
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078253193916527-4">
                 4
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078253193916527-5">
                 5
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078253193916527-6">
                 6
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078253193916527-7">
                 7
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078253193916527-8">
                 8
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078253193916527-9">
                 9
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078253193916527-10">
                 10
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078253193916527-11">
                 11
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078253193916527-12">
                 12
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078253193916527-13">
                 13
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078253193916527-14">
                 14
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078253193916527-15">
                 15
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078253193916527-16">
                 16
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078253193916527-17">
                 17
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078253193916527-18">
                 18
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078253193916527-19">
                 19
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078253193916527-20">
                 20
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078253193916527-21">
                 21
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078253193916527-22">
                 22
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078253193916527-23">
                 23
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078253193916527-24">
                 24
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078253193916527-25">
                 25
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078253193916527-26">
                 26
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078253193916527-27">
                 27
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078253193916527-28">
                 28
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078253193916527-29">
                 29
                </div>
               </div>
              </td>
              <td class="crayon-code">
               <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                <div class="crayon-line" id="crayon-57686ae078253193916527-1">
                 <span class="crayon-p">
                  # Code from Chapter 11 of Machine Learning: An Algorithmic Perspective
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078253193916527-2">
                 <span class="crayon-p">
                  # by Stephen Marsland (http://seat.massey.ac.nz/personal/s.r.marsland/MLBook.html)
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078253193916527-3">
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078253193916527-4">
                 <span class="crayon-p">
                  # Gradient Descent using steepest descent
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078253193916527-5">
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078253193916527-6">
                 <span class="crayon-e">
                  from
                 </span>
                 <span class="crayon-e">
                  numpy
                 </span>
                 <span class="crayon-e ">
                  import *
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078253193916527-7">
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078253193916527-8">
                 <span class="crayon-e">
                  def
                 </span>
                 <span class="crayon-e">
                  Jacobian
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  x
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-o">
                  :
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078253193916527-9">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-st">
                  return
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-t">
                  array
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-sy">
                  [
                 </span>
                 <span class="crayon-v">
                  x
                 </span>
                 <span class="crayon-sy">
                  [
                 </span>
                 <span class="crayon-cn">
                  0
                 </span>
                 <span class="crayon-sy">
                  ]
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-cn">
                  0.4
                 </span>
                 <span class="crayon-o">
                  *
                 </span>
                 <span class="crayon-v">
                  x
                 </span>
                 <span class="crayon-sy">
                  [
                 </span>
                 <span class="crayon-cn">
                  1
                 </span>
                 <span class="crayon-sy">
                  ]
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-cn">
                  1.2
                 </span>
                 <span class="crayon-o">
                  *
                 </span>
                 <span class="crayon-v">
                  x
                 </span>
                 <span class="crayon-sy">
                  [
                 </span>
                 <span class="crayon-cn">
                  2
                 </span>
                 <span class="crayon-sy">
                  ]
                 </span>
                 <span class="crayon-sy">
                  ]
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078253193916527-10">
                </div>
                <div class="crayon-line" id="crayon-57686ae078253193916527-11">
                 <span class="crayon-e">
                  def
                 </span>
                 <span class="crayon-e">
                  steepest
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  x0
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-o">
                  :
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078253193916527-12">
                </div>
                <div class="crayon-line" id="crayon-57686ae078253193916527-13">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  i
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-cn">
                  0
                 </span>
                 <span class="crayon-h">
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078253193916527-14">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  iMax
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-cn">
                  10
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078253193916527-15">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  x
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-e">
                  x0
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078253193916527-16">
                 <span class="crayon-e">
                 </span>
                 <span class="crayon-v">
                  Delta
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-cn">
                  1
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078253193916527-17">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  alpha
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-cn">
                  1
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078253193916527-18">
                </div>
                <div class="crayon-line" id="crayon-57686ae078253193916527-19">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-st">
                  while
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  i
                 </span>
                 <span class="crayon-o">
                  &amp;
                 </span>
                 <span class="crayon-v">
                  lt
                 </span>
                 <span class="crayon-sy">
                  ;
                 </span>
                 <span class="crayon-e">
                  iMax
                 </span>
                 <span class="crayon-st">
                  and
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  Delta
                 </span>
                 <span class="crayon-o">
                  &amp;
                 </span>
                 <span class="crayon-v">
                  gt
                 </span>
                 <span class="crayon-sy">
                  ;
                 </span>
                 <span class="crayon-cn">
                  10
                 </span>
                 <span class="crayon-o">
                  *
                 </span>
                 <span class="crayon-o">
                  *
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-o">
                  -
                 </span>
                 <span class="crayon-cn">
                  5
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-o">
                  :
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078253193916527-20">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  p
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  -
                 </span>
                 <span class="crayon-e">
                  Jacobian
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  x
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078253193916527-21">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  xOld
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-i">
                  x
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078253193916527-22">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  x
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  x
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  +
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-e ">
                  alpha*
                 </span>
                 <span class="crayon-i">
                  p
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078253193916527-23">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  Delta
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-e">
                  sum
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  x
                 </span>
                 <span class="crayon-o">
                  -
                 </span>
                 <span class="crayon-v">
                  xOld
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-o">
                  *
                 </span>
                 <span class="crayon-o">
                  *
                 </span>
                 <span class="crayon-cn">
                  2
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078253193916527-24">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-i">
                  print
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-s">
                  'epoch'
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  i
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-s">
                  ':'
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078253193916527-25">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-i">
                  print
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  x
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-s">
                  '\n'
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078253193916527-26">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  i
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  +=
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-cn">
                  1
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078253193916527-27">
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078253193916527-28">
                 <span class="crayon-v">
                  x0
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-t">
                  array
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-sy">
                  [
                 </span>
                 <span class="crayon-o">
                  -
                 </span>
                 <span class="crayon-cn">
                  2
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-cn">
                  2
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-o">
                  -
                 </span>
                 <span class="crayon-cn">
                  2
                 </span>
                 <span class="crayon-sy">
                  ]
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078253193916527-29">
                 <span class="crayon-e">
                  steepest
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  x0
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                </div>
               </div>
              </td>
             </tr>
            </tbody>
           </table>
          </div>
         </div>
         <!-- [Format Time: 0.0320 seconds] -->
        </div>
        <p>
         Steepest gradient 方法得到的是局部最优解，如果目标函数是一个凸优化问题，那么局部最优解就是全局最优解，理想的优化效果如下图，值得注意一点的是，每一次迭代的移动方向都与出发点的等高线垂直：
        </p>
        <p>
        </p>
        <center>
         <img src="http://dataunion.org/wp-content/uploads/2015/03/350px-Gradient_descent.svg_.png"/>
        </center>
        需要指出的是，在某些情况下，最速下降法存在
        <strong>
         锯齿现象
        </strong>
        （ zig-zagging）将会导致收敛速度变慢:
        <p>
        </p>
        <p>
         粗略来讲，在二次函数中，椭球面的形状受 hesse 矩阵的条件数影响，长轴与短轴对应矩阵的最小特征值和最大特征值的方向，其大小与特征值的平方根成反比，最大特征值与最小特征值相差越大，椭球面越扁，那么优化路径需要走很大的弯路，计算效率很低。
        </p>
        <h3>
         2) Newton’s method
        </h3>
        <p>
         在最速下降法中，我们看到，该方法主要利用的是目标函数的局部性质，具有一定的“盲目性”。牛顿法则是利用局部的一阶和二阶偏导信息，推测整个目标函数的形状，进而可以求得出近似函数的全局最小值，然后将当前的最小值设定近似函数的最小值。相比最速下降法，牛顿法带有一定对全局的预测性，收敛性质也更优良。牛顿法的主要推导过程如下：
        </p>
        <p>
         第一步，利用 Taylor 级数求得原目标函数的二阶近似：
        </p>
        <p>
        </p>
        <center>
         <img src="http://latex.codecogs.com/gif.latex?f(\mathbf{x})\approx%20\phi%20(\mathbf{x})=f(\mathbf{x}^{(k)})+\nabla%20f(\mathbf{x}^{(k)})(\mathbf{x}-\mathbf{x}^{(k)})+\frac{1}{2}(\mathbf{x}-\mathbf{x}^{(k)})^T\nabla%20^2f(\mathbf{x}^{(k)})(\mathbf{x}-\mathbf{x}^{(k)})"/>
        </center>
        第二步，把 x 看做自变量， 所有带有 x^k 的项看做常量，令一阶导数为 0 ，即可求近似函数的最小值：
        <p>
        </p>
        <p>
        </p>
        <center>
         <img src="http://latex.codecogs.com/gif.latex?\mathbf{p}_k%20=%20-(\nabla%20^2f(\mathbf{x_k}))^{-1}\nabla%20f(\mathbf{x_k})"/>
        </center>
        即：
        <p>
        </p>
        <p>
        </p>
        <center>
         <img src="http://latex.codecogs.com/gif.latex?\mathbf{p}_k%20=%20-Hesse^{-1}Jacobi"/>
        </center>
        第三步，将当前的最小值设定近似函数的最小值（或者乘以步长)。
        <p>
        </p>
        <p>
         与
         <strong>
          1)
         </strong>
         中优化问题相同，牛顿法的代码如下：
        </p>
        <p>
         <code>
          Newton.py
         </code>
        </p>
        <div class="cnblogs_code">
         <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
         <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-57686ae078265169544385" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
          <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
           <span class="crayon-title">
           </span>
           <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
            <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
             <div class="crayon-button-icon">
             </div>
            </div>
            <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
             <div class="crayon-button-icon">
             </div>
            </div>
            <div class="crayon-button crayon-wrap-button" title="切换自动换行">
             <div class="crayon-button-icon">
             </div>
            </div>
            <div class="crayon-button crayon-expand-button" title="点击展开代码">
             <div class="crayon-button-icon">
             </div>
            </div>
            <div class="crayon-button crayon-copy-button" title="复制代码">
             <div class="crayon-button-icon">
             </div>
            </div>
            <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
             <div class="crayon-button-icon">
             </div>
            </div>
           </div>
          </div>
          <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
          </div>
          <div class="crayon-plain-wrap">
           <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
            # Code from Chapter 11 of Machine Learning: An Algorithmic Perspective
# by Stephen Marsland (http://seat.massey.ac.nz/personal/s.r.marsland/MLBook.html)

# Gradient Descent using Newton's method
from numpy import *

def Jacobian(x):
    return array([x[0], 0.4*x[1], 1.2*x[2]])

def Hessian(x):
    return array([[1,0,0],[0,0.4,0],[0,0,1.2]])

def Newton(x0):

    i = 0
    iMax = 10
    x = x0
    Delta = 1
    alpha = 1
    
    while i&amp;lt;iMax and Delta&amp;gt;10**(-5):
        p = -dot(linalg.inv(Hessian(x)),Jacobian(x))
        xOld = x
        x = x + alpha*p
        Delta = sum((x-xOld)**2)
        i += 1
    print x
    
x0 = array([-2,2,-2])
Newton(x0)
           </textarea>
          </div>
          <div class="crayon-main" style="">
           <table class="crayon-table">
            <tbody>
             <tr class="crayon-row">
              <td class="crayon-nums " data-settings="show">
               <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                <div class="crayon-num" data-line="crayon-57686ae078265169544385-1">
                 1
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078265169544385-2">
                 2
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078265169544385-3">
                 3
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078265169544385-4">
                 4
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078265169544385-5">
                 5
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078265169544385-6">
                 6
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078265169544385-7">
                 7
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078265169544385-8">
                 8
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078265169544385-9">
                 9
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078265169544385-10">
                 10
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078265169544385-11">
                 11
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078265169544385-12">
                 12
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078265169544385-13">
                 13
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078265169544385-14">
                 14
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078265169544385-15">
                 15
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078265169544385-16">
                 16
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078265169544385-17">
                 17
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078265169544385-18">
                 18
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078265169544385-19">
                 19
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078265169544385-20">
                 20
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078265169544385-21">
                 21
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078265169544385-22">
                 22
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078265169544385-23">
                 23
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078265169544385-24">
                 24
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078265169544385-25">
                 25
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078265169544385-26">
                 26
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078265169544385-27">
                 27
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078265169544385-28">
                 28
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078265169544385-29">
                 29
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078265169544385-30">
                 30
                </div>
               </div>
              </td>
              <td class="crayon-code">
               <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                <div class="crayon-line" id="crayon-57686ae078265169544385-1">
                 <span class="crayon-p">
                  # Code from Chapter 11 of Machine Learning: An Algorithmic Perspective
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078265169544385-2">
                 <span class="crayon-p">
                  # by Stephen Marsland (http://seat.massey.ac.nz/personal/s.r.marsland/MLBook.html)
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078265169544385-3">
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078265169544385-4">
                 <span class="crayon-p">
                  # Gradient Descent using Newton's method
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078265169544385-5">
                 <span class="crayon-e">
                  from
                 </span>
                 <span class="crayon-e">
                  numpy
                 </span>
                 <span class="crayon-e ">
                  import *
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078265169544385-6">
                </div>
                <div class="crayon-line" id="crayon-57686ae078265169544385-7">
                 <span class="crayon-e">
                  def
                 </span>
                 <span class="crayon-e">
                  Jacobian
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  x
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-o">
                  :
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078265169544385-8">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-st">
                  return
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-t">
                  array
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-sy">
                  [
                 </span>
                 <span class="crayon-v">
                  x
                 </span>
                 <span class="crayon-sy">
                  [
                 </span>
                 <span class="crayon-cn">
                  0
                 </span>
                 <span class="crayon-sy">
                  ]
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-cn">
                  0.4
                 </span>
                 <span class="crayon-o">
                  *
                 </span>
                 <span class="crayon-v">
                  x
                 </span>
                 <span class="crayon-sy">
                  [
                 </span>
                 <span class="crayon-cn">
                  1
                 </span>
                 <span class="crayon-sy">
                  ]
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-cn">
                  1.2
                 </span>
                 <span class="crayon-o">
                  *
                 </span>
                 <span class="crayon-v">
                  x
                 </span>
                 <span class="crayon-sy">
                  [
                 </span>
                 <span class="crayon-cn">
                  2
                 </span>
                 <span class="crayon-sy">
                  ]
                 </span>
                 <span class="crayon-sy">
                  ]
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078265169544385-9">
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078265169544385-10">
                 <span class="crayon-e">
                  def
                 </span>
                 <span class="crayon-e">
                  Hessian
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  x
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-o">
                  :
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078265169544385-11">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-st">
                  return
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-t">
                  array
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-sy">
                  [
                 </span>
                 <span class="crayon-sy">
                  [
                 </span>
                 <span class="crayon-cn">
                  1
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-cn">
                  0
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-cn">
                  0
                 </span>
                 <span class="crayon-sy">
                  ]
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-sy">
                  [
                 </span>
                 <span class="crayon-cn">
                  0
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-cn">
                  0.4
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-cn">
                  0
                 </span>
                 <span class="crayon-sy">
                  ]
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-sy">
                  [
                 </span>
                 <span class="crayon-cn">
                  0
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-cn">
                  0
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-cn">
                  1.2
                 </span>
                 <span class="crayon-sy">
                  ]
                 </span>
                 <span class="crayon-sy">
                  ]
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078265169544385-12">
                </div>
                <div class="crayon-line" id="crayon-57686ae078265169544385-13">
                 <span class="crayon-e">
                  def
                 </span>
                 <span class="crayon-e">
                  Newton
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  x0
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-o">
                  :
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078265169544385-14">
                </div>
                <div class="crayon-line" id="crayon-57686ae078265169544385-15">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  i
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-cn">
                  0
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078265169544385-16">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  iMax
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-cn">
                  10
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078265169544385-17">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  x
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-e">
                  x0
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078265169544385-18">
                 <span class="crayon-e">
                 </span>
                 <span class="crayon-v">
                  Delta
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-cn">
                  1
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078265169544385-19">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  alpha
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-cn">
                  1
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078265169544385-20">
                 <span class="crayon-h">
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078265169544385-21">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-st">
                  while
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  i
                 </span>
                 <span class="crayon-o">
                  &amp;
                 </span>
                 <span class="crayon-v">
                  lt
                 </span>
                 <span class="crayon-sy">
                  ;
                 </span>
                 <span class="crayon-e">
                  iMax
                 </span>
                 <span class="crayon-st">
                  and
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  Delta
                 </span>
                 <span class="crayon-o">
                  &amp;
                 </span>
                 <span class="crayon-v">
                  gt
                 </span>
                 <span class="crayon-sy">
                  ;
                 </span>
                 <span class="crayon-cn">
                  10
                 </span>
                 <span class="crayon-o">
                  *
                 </span>
                 <span class="crayon-o">
                  *
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-o">
                  -
                 </span>
                 <span class="crayon-cn">
                  5
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-o">
                  :
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078265169544385-22">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  p
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  -
                 </span>
                 <span class="crayon-e">
                  dot
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  linalg
                 </span>
                 <span class="crayon-sy">
                  .
                 </span>
                 <span class="crayon-e">
                  inv
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-e">
                  Hessian
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  x
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-e">
                  Jacobian
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  x
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078265169544385-23">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  xOld
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-i">
                  x
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078265169544385-24">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  x
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  x
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  +
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-e ">
                  alpha*
                 </span>
                 <span class="crayon-i">
                  p
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078265169544385-25">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  Delta
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-e">
                  sum
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  x
                 </span>
                 <span class="crayon-o">
                  -
                 </span>
                 <span class="crayon-v">
                  xOld
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-o">
                  *
                 </span>
                 <span class="crayon-o">
                  *
                 </span>
                 <span class="crayon-cn">
                  2
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078265169544385-26">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  i
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  +=
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-cn">
                  1
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078265169544385-27">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-i">
                  print
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-i">
                  x
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078265169544385-28">
                 <span class="crayon-h">
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078265169544385-29">
                 <span class="crayon-v">
                  x0
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-t">
                  array
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-sy">
                  [
                 </span>
                 <span class="crayon-o">
                  -
                 </span>
                 <span class="crayon-cn">
                  2
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-cn">
                  2
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-o">
                  -
                 </span>
                 <span class="crayon-cn">
                  2
                 </span>
                 <span class="crayon-sy">
                  ]
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078265169544385-30">
                 <span class="crayon-e">
                  Newton
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  x0
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                </div>
               </div>
              </td>
             </tr>
            </tbody>
           </table>
          </div>
         </div>
         <!-- [Format Time: 0.0238 seconds] -->
        </div>
        <p>
         上面例子中由于目标函数是二次凸函数，Taylor 展开等于原函数，所以能一次就求出最优解。
        </p>
        <p>
         牛顿法主要存在的问题是：
        </p>
        <ol>
         <li>
          Hesse 矩阵不可逆时无法计算
         </li>
         <li>
          矩阵的逆计算复杂为 n 的立方，当问题规模比较大时，计算量很大，解决的办法是采用拟牛顿法如 BFGS, L-BFGS, DFP, Broyden’s Algorithm 进行近似。
         </li>
         <li>
          如果初始值离局部极小值太远，Taylor 展开并不能对原函数进行良好的近似
         </li>
        </ol>
        <h3>
         3) Levenberg–Marquardt Algorithm
        </h3>
        <p>
         Levenberg–Marquardt algorithm 能结合以上两种优化方法的优点，并对两者的不足做出改进。与 line search 的方法不同，LMA 属于一种“信赖域法”(trust region)，牛顿法实际上也可以看做一种信赖域法，即利用局部信息对函数进行建模近似，求取局部最小值。所谓的信赖域法，就是从初始点开始，先假设一个可以信赖的最大位移 s（牛顿法里面 s 为无穷大），然后在以当前点为中心，以 s 为半径的区域内，通过寻找目标函数的一个近似函数（二次的）的最优点，来求解得到真正的位移。在得到了位移之后，再计算目标函数值，如果其使目标函数值的下降满足了一定条件，那么就说明这个位移是可靠的，则继续按此规则迭代计算下去；如果其不能使目标函数值的下降满足一定的条件，则应减小信赖域的范围，再重新求解。
        </p>
        <p>
         LMA 最早提出是用来解决最小二乘法曲线拟合的优化问题的，对于随机初始化的已知参数 beta， 求得的目标值为：
        </p>
        <p>
        </p>
        <center>
         <img src="http://dataunion.org/wp-content/uploads/2015/03/daa7c6ab4351902d897ebffe919e84a1.png"/>
        </center>
        对拟合曲线函数进行一阶 Jacobi 矩阵的近似：
        <p>
        </p>
        <p>
        </p>
        <center>
         <img src="http://dataunion.org/wp-content/uploads/2015/03/f7b732e73d075ff6d6f89622a2bc74f5.png"/>
        </center>
        进而推测出 S 函数的周边信息：
        <p>
        </p>
        <p>
        </p>
        <center>
         <img src="http://dataunion.org/wp-content/uploads/2015/03/e62c1c183de6e455f20a6686a11063db.png"/>
        </center>
        位移是多少时得到 S 函数的最小值呢？通过几何的概念，当残差
        <img src="http://latex.codecogs.com/gif.latex?\mathbf{y}-f(\beta)-J\delta"/>
        垂直于 J 矩阵的 span 空间时， S 取得最小（至于为什么？请参考
        <a href="http://www.cnblogs.com/daniel-D/p/3204508.html" target="_blank">
         之前博客
        </a>
        的最后一部分)
        <p>
        </p>
        <p>
        </p>
        <center>
         <img src="http://dataunion.org/wp-content/uploads/2015/03/d3a53790a98a976ea5419852596ac10f.png"/>
        </center>
        我们将这个公式略加修改，加入阻尼系数得到：
        <p>
        </p>
        <p>
        </p>
        <center>
         <img src="http://dataunion.org/wp-content/uploads/2015/03/55f3b2ec29fd3ea2373a51584e139f31.png"/>
        </center>
        就是莱文贝格－马夸特方法。这种方法只计算了一阶偏导，而且不是目标函数的 Jacobia 矩阵，而是拟合函数的 Jacobia 矩阵。当
        <img src="http://latex.codecogs.com/gif.latex?\lambda"/>
        大的时候可信域小，这种算法会接近最速下降法，
        <img src="http://latex.codecogs.com/gif.latex?\lambda"/>
        小的时候可信域大，会接近高斯-牛顿方法。
        <p>
        </p>
        <p>
         算法过程如下：
        </p>
        <ol>
         <li>
          给定一个初识值 x0
         </li>
         <li>
          当
          <img src="http://latex.codecogs.com/gif.latex?\mathbf{J}^T[\mathbf{y-f(\beta)}]%20%3E%20tolerance"/>
          并且没有到达最大迭代次数时
         </li>
         <li>
          重复执行:
          <ul>
           <li>
            算出移动向量
            <img src="http://latex.codecogs.com/gif.latex?\delta"/>
           </li>
           <li>
            计算更新值：
            <img src="http://latex.codecogs.com/gif.latex?\mathbf{x_{new}}=\mathbf{x}+\delta"/>
           </li>
           <li>
            计算目标函数真实减少量与预测减少量的比率
            <img src="http://latex.codecogs.com/gif.latex?\rho"/>
           </li>
           <li>
            if
            <img src="http://latex.codecogs.com/gif.latex?0%20%3C%20\rho%20%3C%200.25"/>
            ，接受更新值
           </li>
           <li>
            else if
            <img src="http://latex.codecogs.com/gif.latex?\rho%20%3E%200.25"/>
            ，说明近似效果很好，接受更新值，扩大可信域（即减小阻尼系数）
           </li>
           <li>
            else: 目标函数在变大，拒绝更新值，减小可信域（即增加阻尼系数）
           </li>
          </ul>
         </li>
         <li>
          直到达到最大迭代次数
         </li>
        </ol>
        <p>
         维基百科在介绍 Gradient descent 时用包含了细长峡谷的 Rosenbrock function
        </p>
        <p>
        </p>
        <center>
         <img src="http://dataunion.org/wp-content/uploads/2015/03/3505ac88ea3c05a47f3c58b7f70dab59.png"/>
        </center>
        展示了 zig-zagging 锯齿现象：
        <p>
        </p>
        <p>
        </p>
        <center>
         <img src="http://dataunion.org/wp-content/uploads/2015/03/400px-Banana-SteepDesc.gif"/>
        </center>
        用 LMA 优化效率如何。套用到我们之前 LMA 公式中，有：
        <p>
        </p>
        <p>
        </p>
        <center>
         <img src="http://latex.codecogs.com/gif.latex?{%20y%20}\quad%20=\quad%200\\%20f(\beta%20)\quad%20=\quad%20(10(x_{%202%20}-x_{%201%20}^{%202%20}),\quad%201-x_{%201%20})^{%20T%20}\\%20J\quad%20=\quad%20\begin{pmatrix}%20-20{%20x%20}_{%201%20}%20&amp;%2010%20\\%20-1%20&amp;%200%20\end{pmatrix}"/>
        </center>
        代码如下：
        <p>
        </p>
        <p>
         <code>
          LevenbergMarquardt.py
         </code>
        </p>
        <div class="cnblogs_code">
         <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
         <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-57686ae078272341464946" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
          <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
           <span class="crayon-title">
           </span>
           <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
            <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
             <div class="crayon-button-icon">
             </div>
            </div>
            <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
             <div class="crayon-button-icon">
             </div>
            </div>
            <div class="crayon-button crayon-wrap-button" title="切换自动换行">
             <div class="crayon-button-icon">
             </div>
            </div>
            <div class="crayon-button crayon-expand-button" title="点击展开代码">
             <div class="crayon-button-icon">
             </div>
            </div>
            <div class="crayon-button crayon-copy-button" title="复制代码">
             <div class="crayon-button-icon">
             </div>
            </div>
            <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
             <div class="crayon-button-icon">
             </div>
            </div>
           </div>
          </div>
          <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
          </div>
          <div class="crayon-plain-wrap">
           <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
            # Code from Chapter 11 of Machine Learning: An Algorithmic Perspective
# by Stephen Marsland (http://seat.massey.ac.nz/personal/s.r.marsland/MLBook.html)

# The Levenberg Marquardt algorithm
from numpy import *

def function(p):
    r = array([10*(p[1]-p[0]**2),(1-p[0])])
    fp = dot(transpose(r),r) #= 100*(p[1]-p[0]**2)**2 + (1-p[0])**2
    J = (array([[-20*p[0],10],[-1,0]]))
    grad = dot(transpose(J),transpose(r))
    return fp,r,grad,J

def lm(p0,tol=10**(-5),maxits=100):
    
    nvars=shape(p0)[0]
    nu=0.01
    p = p0
    fp,r,grad,J = function(p)
    e = sum(dot(transpose(r),r))
    nits = 0
    while nits&amp;lt;maxits and linalg.norm(grad)&amp;gt;tol:
        nits += 1
        fp,r,grad,J = function(p)
        H=dot(transpose(J),J) + nu*eye(nvars)

        pnew = zeros(shape(p))
        nits2 = 0
        while (p!=pnew).all() and nits2&amp;lt;maxits:
            nits2 += 1
            dp,resid,rank,s = linalg.lstsq(H,grad)
            pnew = p - dp
            fpnew,rnew,gradnew,Jnew = function(pnew)
            enew = sum(dot(transpose(rnew),rnew))
            rho = linalg.norm(dot(transpose(r),r)-dot(transpose(rnew),rnew))
            rho /= linalg.norm(dot(transpose(grad),pnew-p))
            
            if rho&amp;gt;0:
                update = 1
                p = pnew
                e = enew
                if rho&amp;gt;0.25:
                    nu=nu/10
            else: 
                nu=nu*10
                update = 0
        print fp, p, e, linalg.norm(grad), nu

p0 = array([-1.92,2])
lm(p0)
           </textarea>
          </div>
          <div class="crayon-main" style="">
           <table class="crayon-table">
            <tbody>
             <tr class="crayon-row">
              <td class="crayon-nums " data-settings="show">
               <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                <div class="crayon-num" data-line="crayon-57686ae078272341464946-1">
                 1
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078272341464946-2">
                 2
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078272341464946-3">
                 3
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078272341464946-4">
                 4
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078272341464946-5">
                 5
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078272341464946-6">
                 6
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078272341464946-7">
                 7
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078272341464946-8">
                 8
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078272341464946-9">
                 9
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078272341464946-10">
                 10
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078272341464946-11">
                 11
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078272341464946-12">
                 12
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078272341464946-13">
                 13
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078272341464946-14">
                 14
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078272341464946-15">
                 15
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078272341464946-16">
                 16
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078272341464946-17">
                 17
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078272341464946-18">
                 18
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078272341464946-19">
                 19
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078272341464946-20">
                 20
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078272341464946-21">
                 21
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078272341464946-22">
                 22
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078272341464946-23">
                 23
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078272341464946-24">
                 24
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078272341464946-25">
                 25
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078272341464946-26">
                 26
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078272341464946-27">
                 27
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078272341464946-28">
                 28
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078272341464946-29">
                 29
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078272341464946-30">
                 30
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078272341464946-31">
                 31
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078272341464946-32">
                 32
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078272341464946-33">
                 33
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078272341464946-34">
                 34
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078272341464946-35">
                 35
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078272341464946-36">
                 36
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078272341464946-37">
                 37
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078272341464946-38">
                 38
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078272341464946-39">
                 39
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078272341464946-40">
                 40
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078272341464946-41">
                 41
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078272341464946-42">
                 42
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078272341464946-43">
                 43
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078272341464946-44">
                 44
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078272341464946-45">
                 45
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078272341464946-46">
                 46
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078272341464946-47">
                 47
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078272341464946-48">
                 48
                </div>
                <div class="crayon-num" data-line="crayon-57686ae078272341464946-49">
                 49
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae078272341464946-50">
                 50
                </div>
               </div>
              </td>
              <td class="crayon-code">
               <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                <div class="crayon-line" id="crayon-57686ae078272341464946-1">
                 <span class="crayon-p">
                  # Code from Chapter 11 of Machine Learning: An Algorithmic Perspective
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078272341464946-2">
                 <span class="crayon-p">
                  # by Stephen Marsland (http://seat.massey.ac.nz/personal/s.r.marsland/MLBook.html)
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078272341464946-3">
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078272341464946-4">
                 <span class="crayon-p">
                  # The Levenberg Marquardt algorithm
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078272341464946-5">
                 <span class="crayon-e">
                  from
                 </span>
                 <span class="crayon-e">
                  numpy
                 </span>
                 <span class="crayon-e ">
                  import *
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078272341464946-6">
                </div>
                <div class="crayon-line" id="crayon-57686ae078272341464946-7">
                 <span class="crayon-e">
                  def
                 </span>
                 <span class="crayon-t">
                  function
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  p
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-o">
                  :
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078272341464946-8">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  r
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-t">
                  array
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-sy">
                  [
                 </span>
                 <span class="crayon-cn">
                  10
                 </span>
                 <span class="crayon-o">
                  *
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  p
                 </span>
                 <span class="crayon-sy">
                  [
                 </span>
                 <span class="crayon-cn">
                  1
                 </span>
                 <span class="crayon-sy">
                  ]
                 </span>
                 <span class="crayon-o">
                  -
                 </span>
                 <span class="crayon-v">
                  p
                 </span>
                 <span class="crayon-sy">
                  [
                 </span>
                 <span class="crayon-cn">
                  0
                 </span>
                 <span class="crayon-sy">
                  ]
                 </span>
                 <span class="crayon-o">
                  *
                 </span>
                 <span class="crayon-o">
                  *
                 </span>
                 <span class="crayon-cn">
                  2
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-cn">
                  1
                 </span>
                 <span class="crayon-o">
                  -
                 </span>
                 <span class="crayon-v">
                  p
                 </span>
                 <span class="crayon-sy">
                  [
                 </span>
                 <span class="crayon-cn">
                  0
                 </span>
                 <span class="crayon-sy">
                  ]
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-sy">
                  ]
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078272341464946-9">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  fp
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-e">
                  dot
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-e">
                  transpose
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  r
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-v">
                  r
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-p">
                  #= 100*(p[1]-p[0]**2)**2 + (1-p[0])**2
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078272341464946-10">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  J
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-t">
                  array
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-sy">
                  [
                 </span>
                 <span class="crayon-sy">
                  [
                 </span>
                 <span class="crayon-o">
                  -
                 </span>
                 <span class="crayon-cn">
                  20
                 </span>
                 <span class="crayon-o">
                  *
                 </span>
                 <span class="crayon-v">
                  p
                 </span>
                 <span class="crayon-sy">
                  [
                 </span>
                 <span class="crayon-cn">
                  0
                 </span>
                 <span class="crayon-sy">
                  ]
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-cn">
                  10
                 </span>
                 <span class="crayon-sy">
                  ]
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-sy">
                  [
                 </span>
                 <span class="crayon-o">
                  -
                 </span>
                 <span class="crayon-cn">
                  1
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-cn">
                  0
                 </span>
                 <span class="crayon-sy">
                  ]
                 </span>
                 <span class="crayon-sy">
                  ]
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078272341464946-11">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  grad
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-e">
                  dot
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-e">
                  transpose
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  J
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-e">
                  transpose
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  r
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078272341464946-12">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-st">
                  return
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  fp
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-v">
                  r
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-v">
                  grad
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-i">
                  J
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078272341464946-13">
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078272341464946-14">
                 <span class="crayon-e">
                  def
                 </span>
                 <span class="crayon-e">
                  lm
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  p0
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-v">
                  tol
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-cn">
                  10
                 </span>
                 <span class="crayon-o">
                  *
                 </span>
                 <span class="crayon-o">
                  *
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-o">
                  -
                 </span>
                 <span class="crayon-cn">
                  5
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-v">
                  maxits
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-cn">
                  100
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-o">
                  :
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078272341464946-15">
                 <span class="crayon-h">
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078272341464946-16">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  nvars
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-e">
                  shape
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  p0
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-sy">
                  [
                 </span>
                 <span class="crayon-cn">
                  0
                 </span>
                 <span class="crayon-sy">
                  ]
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078272341464946-17">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  nu
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-cn">
                  0.01
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078272341464946-18">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  p
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-e">
                  p0
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078272341464946-19">
                 <span class="crayon-e">
                 </span>
                 <span class="crayon-v">
                  fp
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-v">
                  r
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-v">
                  grad
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-v">
                  J
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-t">
                  function
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  p
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078272341464946-20">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  e
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-e">
                  sum
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-e">
                  dot
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-e">
                  transpose
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  r
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-v">
                  r
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078272341464946-21">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  nits
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-cn">
                  0
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078272341464946-22">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-st">
                  while
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  nits
                 </span>
                 <span class="crayon-o">
                  &amp;
                 </span>
                 <span class="crayon-v">
                  lt
                 </span>
                 <span class="crayon-sy">
                  ;
                 </span>
                 <span class="crayon-e">
                  maxits
                 </span>
                 <span class="crayon-st">
                  and
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  linalg
                 </span>
                 <span class="crayon-sy">
                  .
                 </span>
                 <span class="crayon-e">
                  norm
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  grad
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-o">
                  &amp;
                 </span>
                 <span class="crayon-v">
                  gt
                 </span>
                 <span class="crayon-sy">
                  ;
                 </span>
                 <span class="crayon-v">
                  tol
                 </span>
                 <span class="crayon-o">
                  :
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078272341464946-23">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  nits
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  +=
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-cn">
                  1
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078272341464946-24">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  fp
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-v">
                  r
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-v">
                  grad
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-v">
                  J
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-t">
                  function
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  p
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078272341464946-25">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  H
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-e">
                  dot
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-e">
                  transpose
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  J
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-v">
                  J
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  +
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-e ">
                  nu*
                 </span>
                 <span class="crayon-e">
                  eye
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  nvars
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078272341464946-26">
                </div>
                <div class="crayon-line" id="crayon-57686ae078272341464946-27">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  pnew
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-e">
                  zeros
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-e">
                  shape
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  p
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078272341464946-28">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  nits2
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-cn">
                  0
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078272341464946-29">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-st">
                  while
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  p
                 </span>
                 <span class="crayon-o">
                  !=
                 </span>
                 <span class="crayon-v">
                  pnew
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-sy">
                  .
                 </span>
                 <span class="crayon-e">
                  all
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-st">
                  and
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  nits2
                 </span>
                 <span class="crayon-o">
                  &amp;
                 </span>
                 <span class="crayon-v">
                  lt
                 </span>
                 <span class="crayon-sy">
                  ;
                 </span>
                 <span class="crayon-v">
                  maxits
                 </span>
                 <span class="crayon-o">
                  :
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078272341464946-30">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  nits2
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  +=
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-cn">
                  1
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078272341464946-31">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  dp
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-v">
                  resid
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-v">
                  rank
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-v">
                  s
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  linalg
                 </span>
                 <span class="crayon-sy">
                  .
                 </span>
                 <span class="crayon-e">
                  lstsq
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  H
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-v">
                  grad
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078272341464946-32">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  pnew
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  p
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  -
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-e">
                  dp
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078272341464946-33">
                 <span class="crayon-e">
                 </span>
                 <span class="crayon-v">
                  fpnew
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-v">
                  rnew
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-v">
                  gradnew
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-v">
                  Jnew
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-t">
                  function
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  pnew
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078272341464946-34">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  enew
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-e">
                  sum
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-e">
                  dot
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-e">
                  transpose
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  rnew
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-v">
                  rnew
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078272341464946-35">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  rho
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  linalg
                 </span>
                 <span class="crayon-sy">
                  .
                 </span>
                 <span class="crayon-e">
                  norm
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-e">
                  dot
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-e">
                  transpose
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  r
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-v">
                  r
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-o">
                  -
                 </span>
                 <span class="crayon-e">
                  dot
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-e">
                  transpose
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  rnew
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-v">
                  rnew
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078272341464946-36">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  rho
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  /=
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  linalg
                 </span>
                 <span class="crayon-sy">
                  .
                 </span>
                 <span class="crayon-e">
                  norm
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-e">
                  dot
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-e">
                  transpose
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  grad
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-v">
                  pnew
                 </span>
                 <span class="crayon-o">
                  -
                 </span>
                 <span class="crayon-v">
                  p
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078272341464946-37">
                 <span class="crayon-h">
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078272341464946-38">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-st">
                  if
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  rho
                 </span>
                 <span class="crayon-o">
                  &amp;
                 </span>
                 <span class="crayon-v">
                  gt
                 </span>
                 <span class="crayon-sy">
                  ;
                 </span>
                 <span class="crayon-cn">
                  0
                 </span>
                 <span class="crayon-o">
                  :
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078272341464946-39">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  update
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-cn">
                  1
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078272341464946-40">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  p
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-i">
                  pnew
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078272341464946-41">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  e
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-e">
                  enew
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078272341464946-42">
                 <span class="crayon-e">
                 </span>
                 <span class="crayon-st">
                  if
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  rho
                 </span>
                 <span class="crayon-o">
                  &amp;
                 </span>
                 <span class="crayon-v">
                  gt
                 </span>
                 <span class="crayon-sy">
                  ;
                 </span>
                 <span class="crayon-cn">
                  0.25
                 </span>
                 <span class="crayon-o">
                  :
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078272341464946-43">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  nu
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-v">
                  nu
                 </span>
                 <span class="crayon-o">
                  /
                 </span>
                 <span class="crayon-cn">
                  10
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078272341464946-44">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-st">
                  else
                 </span>
                 <span class="crayon-o">
                  :
                 </span>
                 <span class="crayon-h">
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078272341464946-45">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  nu
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-e ">
                  nu*
                 </span>
                 <span class="crayon-cn">
                  10
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078272341464946-46">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  update
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-cn">
                  0
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae078272341464946-47">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-e">
                  print
                 </span>
                 <span class="crayon-v">
                  fp
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  p
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  e
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  linalg
                 </span>
                 <span class="crayon-sy">
                  .
                 </span>
                 <span class="crayon-e">
                  norm
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  grad
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-e">
                  nu
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078272341464946-48">
                </div>
                <div class="crayon-line" id="crayon-57686ae078272341464946-49">
                 <span class="crayon-v">
                  p0
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-t">
                  array
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-sy">
                  [
                 </span>
                 <span class="crayon-o">
                  -
                 </span>
                 <span class="crayon-cn">
                  1.92
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-cn">
                  2
                 </span>
                 <span class="crayon-sy">
                  ]
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae078272341464946-50">
                 <span class="crayon-e">
                  lm
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  p0
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                </div>
               </div>
              </td>
             </tr>
            </tbody>
           </table>
          </div>
         </div>
         <!-- [Format Time: 0.0384 seconds] -->
        </div>
        <p>
         大概 5 次迭代就可以得到最优解 (1, 1).
        </p>
        <p>
         Levenberg–Marquardt algorithm 对局部极小值很敏感，维基百科举了一个二乘法曲线拟合的例子，当使用不同的初始值时，得到的结果差距很大，我这里也有 python 代码，就不细说了。
        </p>
        <h3>
         4) Conjugate Gradients
        </h3>
        <p>
         共轭梯度法也是优化模型经常经常要用到的一个方法，背后的数学公式和原理稍微复杂一些，光这一个优化方法就可以写一篇很长的博文了，所以这里并不打算详细讲解每一步的推导过程，只简单写一下算法的实现过程。与最速梯度下降的不同，共轭梯度的优点主要体现在选择搜索方向上。在了解共轭梯度法之前，我们首先简单了解一下共轭方向：
        </p>
        <p>
        </p>
        <center>
         <img src="http://dataunion.org/wp-content/uploads/2015/03/2011102021135596.png"/>
        </center>
        共轭方向和马氏距离的定义有类似之处，他们都考虑了全局的数据分布。如上图，d(1) 方向与二次函数的等值线相切，d(1) 的共轭方向 d(2) 则指向椭圆的中心。所以对于二维的二次函数，如果在两个共轭方向上进行一维搜索，经过两次迭代必然达到最小点。前面我们说过，等值线椭圆的形状由 Hesse 矩阵决定，那么，上图的两个方向关于 Hessen 矩阵正交，共轭方向的定义如下：
        <p>
        </p>
        <p>
        </p>
        <center>
         <img src="http://latex.codecogs.com/gif.latex?\mathbf{d}^{(1)T}\mathbf{Ad}^{(2)}%20=%200}"/>
        </center>
        如果椭圆是一个正圆， Hessen 矩阵是一个单位矩阵，上面等价于欧几里得空间中的正交。
        <p>
        </p>
        <p>
         在优化过程中，如果我们确定了移动方向（GD：垂直于等值线，CG：共轭方向），然后在该方向上搜索极小值点（恰好与该处的等值线相切），然后移动到最小值点，重复以上过程，那么 Gradient Descent 和 Conjugate gradient descent 的优化过程可以用下图的
         <strong>
          绿线
         </strong>
         与
         <strong>
          红线
         </strong>
         表示：
        </p>
        <p>
        </p>
        <center>
         <img src="http://dataunion.org/wp-content/uploads/2015/03/220px-Conjugate_gradient_illustration.svg_.png"/>
        </center>
        讲了这么多，共轭梯度算法究竟是如何算的呢？
        <p>
        </p>
        <ol>
         <li>
          给定一个出发点 x0 和一个停止参数 e, 第一次移动方向为
          <strong>
           最速下降方向
          </strong>
          :
          <img src="http://latex.codecogs.com/gif.latex?\mathbf{p}_0%20=%20-\nabla%20f(\mathbf{x})}"/>
         </li>
         <li>
          <img src="http://latex.codecogs.com/gif.latex?\mathbf{p}_{new}%20=%20\mathbf{p}_0"/>
         </li>
         <li>
          while
          <img src="http://latex.codecogs.com/gif.latex?\mathbf{p}_{new}%20%3E%20e^2\mathbf{p}_0"/>
          :
          <ul>
           <li>
            用 Newton-Raphson 迭代计算移动距离，以便在该搜索方向移动到极小，公式就不写了，具体思路就是利用一阶梯度的信息向极小值点跳跃搜索
           </li>
           <li>
            移动当前的优化解 x：
            <img src="http://latex.codecogs.com/gif.latex?\mathbf{x}_{new}%20=%20\mathbf{x}%20+%20\alpha%20_k\mathbf{p}"/>
           </li>
           <li>
            用 Gram-Schmidt 方法构造下一个共轭方向，即
            <img src="http://latex.codecogs.com/gif.latex?\mathbf{p}_{new}%20=%20\nabla%20f(\mathbf{x}_{new})%20+%20\beta%20_{k+1}\mathbf{p}"/>
            , 按照
            <img src="http://latex.codecogs.com/gif.latex?\beta"/>
            的确定公式又可以分为 FR 方法和 PR 和 HS 等。
           </li>
          </ul>
         </li>
        </ol>
        <p>
         在很多的资料中，介绍共轭梯度法都举了一个求线性方程组 Ax = b 近似解的例子，实际上就相当于这里所说的
         <img src="http://latex.codecogs.com/gif.latex?\min%20f(\mathbf{x})%20=%20\frac{1}{2}||\mathbf{Ax-b}||^2"/>
        </p>
        <p>
         还是用最开始的目标函数
         <img src="http://latex.codecogs.com/gif.latex?f(\mathbf{x})=0.5x_1^2%20+%200.2x_2^2%20+%200.6x_3^2"/>
         来编写共轭梯度法的优化代码：
        </p>
        <p>
        </p>
        <div class="cnblogs_code">
         <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
         <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-57686ae07827e959342953" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
          <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
           <span class="crayon-title">
           </span>
           <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
            <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
             <div class="crayon-button-icon">
             </div>
            </div>
            <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
             <div class="crayon-button-icon">
             </div>
            </div>
            <div class="crayon-button crayon-wrap-button" title="切换自动换行">
             <div class="crayon-button-icon">
             </div>
            </div>
            <div class="crayon-button crayon-expand-button" title="点击展开代码">
             <div class="crayon-button-icon">
             </div>
            </div>
            <div class="crayon-button crayon-copy-button" title="复制代码">
             <div class="crayon-button-icon">
             </div>
            </div>
            <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
             <div class="crayon-button-icon">
             </div>
            </div>
           </div>
          </div>
          <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
          </div>
          <div class="crayon-plain-wrap">
           <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
            # Code from Chapter 11 of Machine Learning: An Algorithmic Perspective
# by Stephen Marsland (http://seat.massey.ac.nz/personal/s.r.marsland/MLBook.html)

# The conjugate gradients algorithm

from numpy import *

def Jacobian(x):
    #return array([.4*x[0],2*x[1]])
    return array([x[0], 0.4*x[1], 1.2*x[2]])

def Hessian(x):
    #return array([[.2,0],[0,1]])
    return array([[1,0,0],[0,0.4,0],[0,0,1.2]])

def CG(x0):

    i=0
    k=0

    r = -Jacobian(x0)
    p=r

    betaTop = dot(r.transpose(),r)
    beta0 = betaTop

    iMax = 3
    epsilon = 10**(-2)
    jMax = 5

    # Restart every nDim iterations
    nRestart = shape(x0)[0]
    x = x0

    while i &amp;lt; iMax and betaTop &amp;gt; epsilon**2*beta0:
        j=0
        dp = dot(p.transpose(),p)
        alpha = (epsilon+1)**2
        # Newton-Raphson iteration
        while j &amp;lt; jMax and alpha**2 * dp &amp;gt; epsilon**2:
            # Line search
            alpha = -dot(Jacobian(x).transpose(),p) / (dot(p.transpose(),dot(Hessian(x),p)))
            print "N-R",x, alpha, p
            x = x + alpha * p
            j += 1
        print x
        # Now construct beta
        r = -Jacobian(x)
        print "r: ", r
        betaBottom = betaTop
        betaTop = dot(r.transpose(),r)
        beta = betaTop/betaBottom
        print "Beta: ",beta
        # Update the estimate
        p = r + beta*p
        print "p: ",p
        print "----"
        k += 1
        
        if k==nRestart or dot(r.transpose(),p) &amp;lt;= 0:
            p = r
            k = 0
            print "Restarting"
        i +=1

    print x

x0 = array([-2,2,-2])
CG(x0)
           </textarea>
          </div>
          <div class="crayon-main" style="">
           <table class="crayon-table">
            <tbody>
             <tr class="crayon-row">
              <td class="crayon-nums " data-settings="show">
               <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                <div class="crayon-num" data-line="crayon-57686ae07827e959342953-1">
                 1
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae07827e959342953-2">
                 2
                </div>
                <div class="crayon-num" data-line="crayon-57686ae07827e959342953-3">
                 3
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae07827e959342953-4">
                 4
                </div>
                <div class="crayon-num" data-line="crayon-57686ae07827e959342953-5">
                 5
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae07827e959342953-6">
                 6
                </div>
                <div class="crayon-num" data-line="crayon-57686ae07827e959342953-7">
                 7
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae07827e959342953-8">
                 8
                </div>
                <div class="crayon-num" data-line="crayon-57686ae07827e959342953-9">
                 9
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae07827e959342953-10">
                 10
                </div>
                <div class="crayon-num" data-line="crayon-57686ae07827e959342953-11">
                 11
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae07827e959342953-12">
                 12
                </div>
                <div class="crayon-num" data-line="crayon-57686ae07827e959342953-13">
                 13
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae07827e959342953-14">
                 14
                </div>
                <div class="crayon-num" data-line="crayon-57686ae07827e959342953-15">
                 15
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae07827e959342953-16">
                 16
                </div>
                <div class="crayon-num" data-line="crayon-57686ae07827e959342953-17">
                 17
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae07827e959342953-18">
                 18
                </div>
                <div class="crayon-num" data-line="crayon-57686ae07827e959342953-19">
                 19
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae07827e959342953-20">
                 20
                </div>
                <div class="crayon-num" data-line="crayon-57686ae07827e959342953-21">
                 21
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae07827e959342953-22">
                 22
                </div>
                <div class="crayon-num" data-line="crayon-57686ae07827e959342953-23">
                 23
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae07827e959342953-24">
                 24
                </div>
                <div class="crayon-num" data-line="crayon-57686ae07827e959342953-25">
                 25
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae07827e959342953-26">
                 26
                </div>
                <div class="crayon-num" data-line="crayon-57686ae07827e959342953-27">
                 27
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae07827e959342953-28">
                 28
                </div>
                <div class="crayon-num" data-line="crayon-57686ae07827e959342953-29">
                 29
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae07827e959342953-30">
                 30
                </div>
                <div class="crayon-num" data-line="crayon-57686ae07827e959342953-31">
                 31
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae07827e959342953-32">
                 32
                </div>
                <div class="crayon-num" data-line="crayon-57686ae07827e959342953-33">
                 33
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae07827e959342953-34">
                 34
                </div>
                <div class="crayon-num" data-line="crayon-57686ae07827e959342953-35">
                 35
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae07827e959342953-36">
                 36
                </div>
                <div class="crayon-num" data-line="crayon-57686ae07827e959342953-37">
                 37
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae07827e959342953-38">
                 38
                </div>
                <div class="crayon-num" data-line="crayon-57686ae07827e959342953-39">
                 39
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae07827e959342953-40">
                 40
                </div>
                <div class="crayon-num" data-line="crayon-57686ae07827e959342953-41">
                 41
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae07827e959342953-42">
                 42
                </div>
                <div class="crayon-num" data-line="crayon-57686ae07827e959342953-43">
                 43
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae07827e959342953-44">
                 44
                </div>
                <div class="crayon-num" data-line="crayon-57686ae07827e959342953-45">
                 45
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae07827e959342953-46">
                 46
                </div>
                <div class="crayon-num" data-line="crayon-57686ae07827e959342953-47">
                 47
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae07827e959342953-48">
                 48
                </div>
                <div class="crayon-num" data-line="crayon-57686ae07827e959342953-49">
                 49
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae07827e959342953-50">
                 50
                </div>
                <div class="crayon-num" data-line="crayon-57686ae07827e959342953-51">
                 51
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae07827e959342953-52">
                 52
                </div>
                <div class="crayon-num" data-line="crayon-57686ae07827e959342953-53">
                 53
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae07827e959342953-54">
                 54
                </div>
                <div class="crayon-num" data-line="crayon-57686ae07827e959342953-55">
                 55
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae07827e959342953-56">
                 56
                </div>
                <div class="crayon-num" data-line="crayon-57686ae07827e959342953-57">
                 57
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae07827e959342953-58">
                 58
                </div>
                <div class="crayon-num" data-line="crayon-57686ae07827e959342953-59">
                 59
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae07827e959342953-60">
                 60
                </div>
                <div class="crayon-num" data-line="crayon-57686ae07827e959342953-61">
                 61
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae07827e959342953-62">
                 62
                </div>
                <div class="crayon-num" data-line="crayon-57686ae07827e959342953-63">
                 63
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae07827e959342953-64">
                 64
                </div>
                <div class="crayon-num" data-line="crayon-57686ae07827e959342953-65">
                 65
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae07827e959342953-66">
                 66
                </div>
                <div class="crayon-num" data-line="crayon-57686ae07827e959342953-67">
                 67
                </div>
                <div class="crayon-num crayon-striped-num" data-line="crayon-57686ae07827e959342953-68">
                 68
                </div>
                <div class="crayon-num" data-line="crayon-57686ae07827e959342953-69">
                 69
                </div>
               </div>
              </td>
              <td class="crayon-code">
               <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                <div class="crayon-line" id="crayon-57686ae07827e959342953-1">
                 <span class="crayon-p">
                  # Code from Chapter 11 of Machine Learning: An Algorithmic Perspective
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae07827e959342953-2">
                 <span class="crayon-p">
                  # by Stephen Marsland (http://seat.massey.ac.nz/personal/s.r.marsland/MLBook.html)
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae07827e959342953-3">
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae07827e959342953-4">
                 <span class="crayon-p">
                  # The conjugate gradients algorithm
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae07827e959342953-5">
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae07827e959342953-6">
                 <span class="crayon-e">
                  from
                 </span>
                 <span class="crayon-e">
                  numpy
                 </span>
                 <span class="crayon-e ">
                  import *
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae07827e959342953-7">
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae07827e959342953-8">
                 <span class="crayon-e">
                  def
                 </span>
                 <span class="crayon-e">
                  Jacobian
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  x
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-o">
                  :
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae07827e959342953-9">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-p">
                  #return array([.4*x[0],2*x[1]])
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae07827e959342953-10">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-st">
                  return
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-t">
                  array
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-sy">
                  [
                 </span>
                 <span class="crayon-v">
                  x
                 </span>
                 <span class="crayon-sy">
                  [
                 </span>
                 <span class="crayon-cn">
                  0
                 </span>
                 <span class="crayon-sy">
                  ]
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-cn">
                  0.4
                 </span>
                 <span class="crayon-o">
                  *
                 </span>
                 <span class="crayon-v">
                  x
                 </span>
                 <span class="crayon-sy">
                  [
                 </span>
                 <span class="crayon-cn">
                  1
                 </span>
                 <span class="crayon-sy">
                  ]
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-cn">
                  1.2
                 </span>
                 <span class="crayon-o">
                  *
                 </span>
                 <span class="crayon-v">
                  x
                 </span>
                 <span class="crayon-sy">
                  [
                 </span>
                 <span class="crayon-cn">
                  2
                 </span>
                 <span class="crayon-sy">
                  ]
                 </span>
                 <span class="crayon-sy">
                  ]
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae07827e959342953-11">
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae07827e959342953-12">
                 <span class="crayon-e">
                  def
                 </span>
                 <span class="crayon-e">
                  Hessian
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  x
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-o">
                  :
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae07827e959342953-13">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-p">
                  #return array([[.2,0],[0,1]])
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae07827e959342953-14">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-st">
                  return
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-t">
                  array
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-sy">
                  [
                 </span>
                 <span class="crayon-sy">
                  [
                 </span>
                 <span class="crayon-cn">
                  1
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-cn">
                  0
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-cn">
                  0
                 </span>
                 <span class="crayon-sy">
                  ]
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-sy">
                  [
                 </span>
                 <span class="crayon-cn">
                  0
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-cn">
                  0.4
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-cn">
                  0
                 </span>
                 <span class="crayon-sy">
                  ]
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-sy">
                  [
                 </span>
                 <span class="crayon-cn">
                  0
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-cn">
                  0
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-cn">
                  1.2
                 </span>
                 <span class="crayon-sy">
                  ]
                 </span>
                 <span class="crayon-sy">
                  ]
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae07827e959342953-15">
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae07827e959342953-16">
                 <span class="crayon-e">
                  def
                 </span>
                 <span class="crayon-e">
                  CG
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  x0
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-o">
                  :
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae07827e959342953-17">
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae07827e959342953-18">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  i
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-cn">
                  0
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae07827e959342953-19">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  k
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-cn">
                  0
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae07827e959342953-20">
                </div>
                <div class="crayon-line" id="crayon-57686ae07827e959342953-21">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  r
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  -
                 </span>
                 <span class="crayon-e">
                  Jacobian
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  x0
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae07827e959342953-22">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  p
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-i">
                  r
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae07827e959342953-23">
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae07827e959342953-24">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  betaTop
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-e">
                  dot
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  r
                 </span>
                 <span class="crayon-sy">
                  .
                 </span>
                 <span class="crayon-e">
                  transpose
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-v">
                  r
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae07827e959342953-25">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  beta0
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-e">
                  betaTop
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae07827e959342953-26">
                </div>
                <div class="crayon-line" id="crayon-57686ae07827e959342953-27">
                 <span class="crayon-e">
                 </span>
                 <span class="crayon-v">
                  iMax
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-cn">
                  3
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae07827e959342953-28">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  epsilon
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-cn">
                  10
                 </span>
                 <span class="crayon-o">
                  *
                 </span>
                 <span class="crayon-o">
                  *
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-o">
                  -
                 </span>
                 <span class="crayon-cn">
                  2
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae07827e959342953-29">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  jMax
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-cn">
                  5
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae07827e959342953-30">
                </div>
                <div class="crayon-line" id="crayon-57686ae07827e959342953-31">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-p">
                  # Restart every nDim iterations
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae07827e959342953-32">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  nRestart
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-e">
                  shape
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  x0
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-sy">
                  [
                 </span>
                 <span class="crayon-cn">
                  0
                 </span>
                 <span class="crayon-sy">
                  ]
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae07827e959342953-33">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  x
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-e">
                  x0
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae07827e959342953-34">
                </div>
                <div class="crayon-line" id="crayon-57686ae07827e959342953-35">
                 <span class="crayon-e">
                 </span>
                 <span class="crayon-st">
                  while
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  i
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  &amp;
                 </span>
                 <span class="crayon-v">
                  lt
                 </span>
                 <span class="crayon-sy">
                  ;
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-e">
                  iMax
                 </span>
                 <span class="crayon-st">
                  and
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  betaTop
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  &amp;
                 </span>
                 <span class="crayon-v">
                  gt
                 </span>
                 <span class="crayon-sy">
                  ;
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-e ">
                  epsilon*
                 </span>
                 <span class="crayon-o">
                  *
                 </span>
                 <span class="crayon-cn">
                  2
                 </span>
                 <span class="crayon-o">
                  *
                 </span>
                 <span class="crayon-v">
                  beta0
                 </span>
                 <span class="crayon-o">
                  :
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae07827e959342953-36">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  j
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-cn">
                  0
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae07827e959342953-37">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  dp
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-e">
                  dot
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  p
                 </span>
                 <span class="crayon-sy">
                  .
                 </span>
                 <span class="crayon-e">
                  transpose
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-v">
                  p
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae07827e959342953-38">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  alpha
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  epsilon
                 </span>
                 <span class="crayon-o">
                  +
                 </span>
                 <span class="crayon-cn">
                  1
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-o">
                  *
                 </span>
                 <span class="crayon-o">
                  *
                 </span>
                 <span class="crayon-cn">
                  2
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae07827e959342953-39">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-p">
                  # Newton-Raphson iteration
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae07827e959342953-40">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-st">
                  while
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  j
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  &amp;
                 </span>
                 <span class="crayon-v">
                  lt
                 </span>
                 <span class="crayon-sy">
                  ;
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-e">
                  jMax
                 </span>
                 <span class="crayon-st">
                  and
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-e ">
                  alpha*
                 </span>
                 <span class="crayon-o">
                  *
                 </span>
                 <span class="crayon-cn">
                  2
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  *
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  dp
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  &amp;
                 </span>
                 <span class="crayon-v">
                  gt
                 </span>
                 <span class="crayon-sy">
                  ;
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-e ">
                  epsilon*
                 </span>
                 <span class="crayon-o">
                  *
                 </span>
                 <span class="crayon-cn">
                  2
                 </span>
                 <span class="crayon-o">
                  :
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae07827e959342953-41">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-p">
                  # Line search
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae07827e959342953-42">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  alpha
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  -
                 </span>
                 <span class="crayon-e">
                  dot
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-e">
                  Jacobian
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  x
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-sy">
                  .
                 </span>
                 <span class="crayon-e">
                  transpose
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-v">
                  p
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  /
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-e">
                  dot
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  p
                 </span>
                 <span class="crayon-sy">
                  .
                 </span>
                 <span class="crayon-e">
                  transpose
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-e">
                  dot
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-e">
                  Hessian
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  x
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-v">
                  p
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae07827e959342953-43">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-i">
                  print
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-s">
                  "N-R"
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-v">
                  x
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  alpha
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-i">
                  p
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae07827e959342953-44">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  x
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  x
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  +
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-e ">
                  alpha *
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-i">
                  p
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae07827e959342953-45">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  j
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  +=
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-cn">
                  1
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae07827e959342953-46">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-i">
                  print
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  x
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae07827e959342953-47">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-p">
                  # Now construct beta
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae07827e959342953-48">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  r
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  -
                 </span>
                 <span class="crayon-e">
                  Jacobian
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  x
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae07827e959342953-49">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-i">
                  print
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-s">
                  "r: "
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-i">
                  r
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae07827e959342953-50">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  betaBottom
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-e">
                  betaTop
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae07827e959342953-51">
                 <span class="crayon-e">
                 </span>
                 <span class="crayon-v">
                  betaTop
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-e">
                  dot
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  r
                 </span>
                 <span class="crayon-sy">
                  .
                 </span>
                 <span class="crayon-e">
                  transpose
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-v">
                  r
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae07827e959342953-52">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  beta
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  betaTop
                 </span>
                 <span class="crayon-o">
                  /
                 </span>
                 <span class="crayon-e">
                  betaBottom
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae07827e959342953-53">
                 <span class="crayon-e">
                 </span>
                 <span class="crayon-i">
                  print
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-s">
                  "Beta: "
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-v">
                  beta
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae07827e959342953-54">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-p">
                  # Update the estimate
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae07827e959342953-55">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  p
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  r
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  +
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-e ">
                  beta*
                 </span>
                 <span class="crayon-i">
                  p
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae07827e959342953-56">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-i">
                  print
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-s">
                  "p: "
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-i">
                  p
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae07827e959342953-57">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-i">
                  print
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-s">
                  "----"
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae07827e959342953-58">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  k
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  +=
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-cn">
                  1
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae07827e959342953-59">
                 <span class="crayon-h">
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae07827e959342953-60">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-st">
                  if
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  k
                 </span>
                 <span class="crayon-o">
                  ==
                 </span>
                 <span class="crayon-e">
                  nRestart
                 </span>
                 <span class="crayon-st">
                  or
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-e">
                  dot
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  r
                 </span>
                 <span class="crayon-sy">
                  .
                 </span>
                 <span class="crayon-e">
                  transpose
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-v">
                  p
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  &amp;
                 </span>
                 <span class="crayon-v">
                  lt
                 </span>
                 <span class="crayon-sy">
                  ;
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-cn">
                  0
                 </span>
                 <span class="crayon-o">
                  :
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae07827e959342953-61">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  p
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-i">
                  r
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae07827e959342953-62">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  k
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-cn">
                  0
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae07827e959342953-63">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-i">
                  print
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-s">
                  "Restarting"
                 </span>
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae07827e959342953-64">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-v">
                  i
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  +=
                 </span>
                 <span class="crayon-cn">
                  1
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae07827e959342953-65">
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae07827e959342953-66">
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-i">
                  print
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-i">
                  x
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae07827e959342953-67">
                </div>
                <div class="crayon-line crayon-striped-line" id="crayon-57686ae07827e959342953-68">
                 <span class="crayon-v">
                  x0
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-o">
                  =
                 </span>
                 <span class="crayon-h">
                 </span>
                 <span class="crayon-t">
                  array
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-sy">
                  [
                 </span>
                 <span class="crayon-o">
                  -
                 </span>
                 <span class="crayon-cn">
                  2
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-cn">
                  2
                 </span>
                 <span class="crayon-sy">
                  ,
                 </span>
                 <span class="crayon-o">
                  -
                 </span>
                 <span class="crayon-cn">
                  2
                 </span>
                 <span class="crayon-sy">
                  ]
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                </div>
                <div class="crayon-line" id="crayon-57686ae07827e959342953-69">
                 <span class="crayon-e">
                  CG
                 </span>
                 <span class="crayon-sy">
                  (
                 </span>
                 <span class="crayon-v">
                  x0
                 </span>
                 <span class="crayon-sy">
                  )
                 </span>
                </div>
               </div>
              </td>
             </tr>
            </tbody>
           </table>
          </div>
         </div>
         <!-- [Format Time: 0.0304 seconds] -->
        </div>
        <p>
        </p>
        <hr/>
        <p>
         参考资料：
        </p>
        <p>
         [1] Machine Learning: An Algorithmic Perspective, chapter 11
         <br/>
         [2] 最优化理论与算法（第2版），陈宝林
         <br/>
         [3] wikipedia
        </p>
       </div>
       <div id="MySignature">
        文章出处：http://www.cnblogs.com/daniel-D/
       </div>
       <div>
       </div>
       <div>
       </div>
      </div>
      <div>
       <strong>
        注：转载文章均来自于公开网络，仅供学习使用，不会用于任何商业用途，如果侵犯到原作者的权益，请您与我们联系删除或者授权事宜，联系邮箱：contact@dataunion.org。转载数盟网站文章请注明原文章作者，否则产生的任何版权纠纷与数盟无关。
       </strong>
      </div>
      <!--content_text-->
      <div class="fenxian">
       <!-- JiaThis Button BEGIN -->
       <div class="jiathis_style_32x32">
        <p class="jiathis_button_weixin">
        </p>
        <p class="jiathis_button_tsina">
        </p>
        <p class="jiathis_button_qzone">
        </p>
        <p class="jiathis_button_cqq">
        </p>
        <p class="jiathis_button_tumblr">
        </p>
        <a class="jiathis jiathis_txt jtico jtico_jiathis" href="http://www.jiathis.com/share" target="_blank">
        </a>
        <p class="jiathis_counter_style">
        </p>
       </div>
       <!-- JiaThis Button END -->
      </div>
     </article>
     <!--content-->
     <!--相关文章-->
     <div class="xianguan">
      <div class="xianguantitle">
       相关文章！
      </div>
      <ul class="pic">
       <li>
        <a href="http://dataunion.org/20758.html">
         <img src="http://dataunion.org/wp-content/uploads/2015/09/t012aec1078cd96f01f-300x199.jpg"/>
        </a>
        <a class="link" href="http://dataunion.org/20758.html" rel="bookmark" title="推荐一些从零开始学习机器学习算法的方法">
         推荐一些从零开始学习机器学习算法的方法
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/20714.html">
         <img src="http://dataunion.org/wp-content/uploads/2015/09/t011650727f6392a3f5-300x197.jpg"/>
        </a>
        <a class="link" href="http://dataunion.org/20714.html" rel="bookmark" title="寻找最优参数解：最速下降法，牛顿下降法，阻尼牛顿法，拟牛顿法DFP/BFGS">
         寻找最优参数解：最速下降法，牛顿下降法，阻尼牛顿法，拟牛顿法DFP/BFGS
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/20636.html">
         <img src="http://dataunion.org/wp-content/uploads/2015/09/t01034a6a3b660f0c04_副本-300x203.jpg"/>
        </a>
        <a class="link" href="http://dataunion.org/20636.html" rel="bookmark" title="机器学习开发者的现代化攻略：对统计学微积分Say NO！">
         机器学习开发者的现代化攻略：对统计学微积分Say NO！
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/20632.html">
         <img src="http://dataunion.org/wp-content/uploads/2015/08/t01e4f9fea2b69f4b1f-300x189.jpg"/>
        </a>
        <a class="link" href="http://dataunion.org/20632.html" rel="bookmark" title="机器学习到底适合哪些人群学习？">
         机器学习到底适合哪些人群学习？
        </a>
       </li>
      </ul>
     </div>
     <!--相关文章-->
     <div class="comment" id="comments">
      <!-- You can start editing here. -->
      <!-- If comments are open, but there are no comments. -->
      <div class="title">
       期待你一针见血的评论，Come on！
      </div>
      <div id="respond">
       <p>
        不用想啦，马上
        <a href="http://dataunion.org/wp-login.php?redirect_to=http%3A%2F%2Fdataunion.org%2F11752.html">
         "登录"
        </a>
        发表自已的想法.
       </p>
      </div>
     </div>
     <!-- .nav-single -->
    </div>
    <!--Container End-->
    <aside id="sitebar">
     <div class="sitebar_list2">
      <div class="wptag">
       <span class="tagtitle">
        热门标签+
       </span>
       <div class="tagg">
        <ul class="menu" id="menu-%e5%8f%8b%e6%83%85%e9%93%be%e6%8e%a5">
         <li class="menu-item menu-item-type-custom menu-item-object-custom menu-item-1605" id="menu-item-1605">
          <a href="http://taidizh.com/">
           泰迪智慧
          </a>
         </li>
         <li class="menu-item menu-item-type-custom menu-item-object-custom menu-item-20884" id="menu-item-20884">
          <a href="http://www.transwarp.cn/">
           星环科技
          </a>
         </li>
         <li class="menu-item menu-item-type-custom menu-item-object-custom menu-item-3538" id="menu-item-3538">
          <a href="http://datall.org/">
           珈和遥感
          </a>
         </li>
         <li class="menu-item menu-item-type-custom menu-item-object-custom menu-item-20888" id="menu-item-20888">
          <a href="http://www.chinahadoop.cn/">
           小象学院
          </a>
         </li>
        </ul>
       </div>
      </div>
     </div>
     <div class="sitebar_list">
      <div class="textwidget">
       <div align="center">
        <a href="http://study.163.com/course/courseMain.htm?courseId=991022" target="_blank">
         <img src="http://dataunion.org/wp-content/uploads/2016/03/dv.jpg"/>
        </a>
       </div>
      </div>
     </div>
     <div class="sitebar_list">
      <h4 class="sitebar_title">
       文章分类
      </h4>
      <div class="tagcloud">
       <a class="tag-link-44" href="http://dataunion.org/category/industry/demo" style="font-size: 10.204724409449pt;" title="4个话题">
        Demo展示
       </a>
       <a class="tag-link-31" href="http://dataunion.org/category/experts" style="font-size: 15.826771653543pt;" title="52个话题">
        专家团队
       </a>
       <a class="tag-link-870" href="http://dataunion.org/category/tech/ai" style="font-size: 19.795275590551pt;" title="273个话题">
        人工智能
       </a>
       <a class="tag-link-488" href="http://dataunion.org/category/%e5%8a%a0%e5%85%a5%e6%95%b0%e7%9b%9f" style="font-size: 8pt;" title="1个话题">
        加入数盟
       </a>
       <a class="tag-link-869" href="http://dataunion.org/category/tech/viz" style="font-size: 17.204724409449pt;" title="93个话题">
        可视化
       </a>
       <a class="tag-link-30" href="http://dataunion.org/category/partners" style="font-size: 10.645669291339pt;" title="5个话题">
        合作伙伴
       </a>
       <a class="tag-link-889" href="http://dataunion.org/category/parterc" style="font-size: 11.582677165354pt;" title="8个话题">
        合作会议
       </a>
       <a class="tag-link-104" href="http://dataunion.org/category/books" style="font-size: 12.96062992126pt;" title="15个话题">
        图书
       </a>
       <a class="tag-link-220" href="http://dataunion.org/category/tech/base" style="font-size: 19.850393700787pt;" title="281个话题">
        基础架构
       </a>
       <a class="tag-link-219" href="http://dataunion.org/category/tech/analysis" style="font-size: 19.409448818898pt;" title="232个话题">
        数据分析
       </a>
       <a class="tag-link-887" href="http://dataunion.org/category/tech/dm" style="font-size: 13.291338582677pt;" title="17个话题">
        数据挖掘
       </a>
       <a class="tag-link-34" href="http://dataunion.org/category/tech" style="font-size: 20.732283464567pt;" title="404个话题">
        文章
       </a>
       <a class="tag-link-1" href="http://dataunion.org/category/uncategorized" style="font-size: 22pt;" title="693个话题">
        未分类
       </a>
       <a class="tag-link-4" href="http://dataunion.org/category/events" style="font-size: 14.503937007874pt;" title="29个话题">
        活动
       </a>
       <a class="tag-link-890" href="http://dataunion.org/category/tech/%e6%b7%b1%e5%ba%a6%e5%ad%a6%e4%b9%a0" style="font-size: 10.204724409449pt;" title="4个话题">
        深度学习
       </a>
       <a class="tag-link-221" href="http://dataunion.org/category/tech/devl" style="font-size: 18.968503937008pt;" title="193个话题">
        编程语言
       </a>
       <a class="tag-link-888" href="http://dataunion.org/category/career" style="font-size: 15.661417322835pt;" title="48个话题">
        职业规划
       </a>
       <a class="tag-link-5" href="http://dataunion.org/category/jobs" style="font-size: 14.11811023622pt;" title="25个话题">
        职位
       </a>
       <a class="tag-link-871" href="http://dataunion.org/category/industry" style="font-size: 15.716535433071pt;" title="49个话题">
        行业
       </a>
       <a class="tag-link-613" href="http://dataunion.org/category/industry/case" style="font-size: 16.984251968504pt;" title="84个话题">
        行业应用
       </a>
       <a class="tag-link-885" href="http://dataunion.org/category/industry/news" style="font-size: 17.425196850394pt;" title="102个话题">
        行业资讯
       </a>
       <a class="tag-link-10" href="http://dataunion.org/category/training" style="font-size: 14.228346456693pt;" title="26个话题">
        课程
       </a>
       <a class="tag-link-16" href="http://dataunion.org/category/sources" style="font-size: 15.661417322835pt;" title="48个话题">
        资源
       </a>
      </div>
     </div>
     <div class="sitebar_list">
      <h4 class="sitebar_title">
       功能
      </h4>
      <ul>
       <li>
        <a href="http://dataunion.org/wp-login.php?action=register">
         注册
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/wp-login.php">
         登录
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/feed">
         文章
         <abbr title="Really Simple Syndication">
          RSS
         </abbr>
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/comments/feed">
         评论
         <abbr title="Really Simple Syndication">
          RSS
         </abbr>
        </a>
       </li>
       <li>
        <a href="https://cn.wordpress.org/" title="基于WordPress，一个优美、先进的个人信息发布平台。">
         WordPress.org
        </a>
       </li>
      </ul>
     </div>
    </aside>
    <div class="clear">
    </div>
   </div>
   <!--main-->
   ﻿
   <footer id="dibu">
    <div class="about">
     <div class="right">
      <ul class="menu" id="menu-%e5%ba%95%e9%83%a8%e8%8f%9c%e5%8d%95">
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-18024" id="menu-item-18024">
        <a href="http://dataunion.org/category/partners">
         合作伙伴
        </a>
       </li>
       <li class="menu-item menu-item-type-post_type menu-item-object-page menu-item-20881" id="menu-item-20881">
        <a href="http://dataunion.org/contribute">
         文章投稿
        </a>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-20872" id="menu-item-20872">
        <a href="http://dataunion.org/category/%e5%8a%a0%e5%85%a5%e6%95%b0%e7%9b%9f">
         加入数盟
        </a>
       </li>
       <li class="menu-item menu-item-type-post_type menu-item-object-page menu-item-22441" id="menu-item-22441">
        <a href="http://dataunion.org/f-links">
         友情链接
        </a>
       </li>
       <li class="menu-item menu-item-type-post_type menu-item-object-page menu-item-20874" id="menu-item-20874">
        <a href="http://dataunion.org/aboutus">
         关于数盟
        </a>
       </li>
      </ul>
      <p class="banquan">
       数盟社区        ，
        做最棒的数据科学社区
      </p>
     </div>
     <div class="left">
      <ul class="bottomlist">
       <li>
        <a href="http://weibo.com/DataScientistUnion  " target="_blank" 　title="">
         <img src="http://dataunion.org/wp-content/themes/yzipi/images/weibo.png"/>
        </a>
       </li>
       <li>
        <a class="cd-popup-trigger" href="http://dataunion.org/11752.html#0">
         <img src="http://dataunion.org/wp-content/themes/yzipi/images/weixin.png"/>
        </a>
       </li>
      </ul>
      <div class="cd-popup">
       <div class="cd-popup-container">
        <h1>
         扫描二维码,加微信公众号
        </h1>
        <img src="http://dataunion.org/wp-content/themes/yzipi/images/2014-12-06-1515289049.png"/>
        <a class="cd-popup-close" href="http://dataunion.org/11752.html">
        </a>
       </div>
       <!-- cd-popup-container -->
      </div>
      <!-- cd-popup -->
     </div>
    </div>
    <!--about-->
    <div class="bottom">
     <a href="http://dataunion.org/">
      数盟社区
     </a>
     <a href="http://www.miitbeian.gov.cn/" rel="external nofollow" target="_blank">
      京ICP备14026740号
     </a>
     联系我们：
     <a href="mailto:contact@dataunion.org" target="_blank">
      contact@dataunion.org
     </a>
     <div class="tongji">
     </div>
     <!--bottom-->
     <div class="scroll" id="scroll" style="display:none;">
      ︿
     </div>
    </div>
   </footer>
   <!--dibu-->
  </div>
 </body>
</html>